{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Kaggle competition: Predicting a Biological Response.\n",
    "\n",
    "Blending {RandomForests, ExtraTrees, GradientBoosting} + stretching to\n",
    "[0,1]. The blending scheme is related to the idea Jose H. Solorzano\n",
    "presented here:\n",
    "http://www.kaggle.com/c/bioresponse/forums/t/1889/question-about-the-process-of-ensemble-learning/10950#post10950\n",
    "'''You can try this: In one of the 5 folds, train the models, then use\n",
    "the results of the models as 'variables' in logistic regression over\n",
    "the validation data of that fold'''. Or at least this is the\n",
    "implementation of my understanding of that idea :-)\n",
    "\n",
    "The predictions are saved in test.csv. The code below created my best\n",
    "submission to the competition:\n",
    "- public score (25%): 0.43464\n",
    "- private score (75%): 0.37751\n",
    "- final rank on the private leaderboard: 17th over 711 teams :-)\n",
    "\n",
    "Note: if you increase the number of estimators of the classifiers,\n",
    "e.g. n_estimators=1000, you get a better score/rank on the private\n",
    "test set.\n",
    "\n",
    "Copyright 2012, Emanuele Olivetti.\n",
    "BSD license, 3 clauses.\n",
    "\"\"\"\n",
    "\n",
    "from __future__ import division\n",
    "import numpy as np\n",
    "import load_data\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "\n",
    "def logloss(attempt, actual, epsilon=1.0e-15):\n",
    "    \"\"\"Logloss, i.e. the score of the bioresponse competition.\n",
    "    \"\"\"\n",
    "    attempt = np.clip(attempt, epsilon, 1.0-epsilon)\n",
    "    return - np.mean(actual * np.log(attempt) +\n",
    "                     (1.0 - actual) * np.log(1.0 - attempt))\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "\n",
    "    np.random.seed(0)  # seed to shuffle the train set\n",
    "\n",
    "    n_folds = 10\n",
    "    verbose = True\n",
    "    shuffle = False\n",
    "\n",
    "    X, y, X_submission = load_data.load()\n",
    "\n",
    "    if shuffle:\n",
    "        idx = np.random.permutation(y.size)\n",
    "        X = X[idx]\n",
    "        y = y[idx]\n",
    "\n",
    "    skf = list(StratifiedKFold(y, n_folds))\n",
    "\n",
    "    clfs = [RandomForestClassifier(n_estimators=100, n_jobs=-1, criterion='gini'),\n",
    "            RandomForestClassifier(n_estimators=100, n_jobs=-1, criterion='entropy'),\n",
    "            ExtraTreesClassifier(n_estimators=100, n_jobs=-1, criterion='gini'),\n",
    "            ExtraTreesClassifier(n_estimators=100, n_jobs=-1, criterion='entropy'),\n",
    "            GradientBoostingClassifier(learning_rate=0.05, subsample=0.5, max_depth=6, n_estimators=50)]\n",
    "\n",
    "    print \"Creating train and test sets for blending.\"\n",
    "\n",
    "    dataset_blend_train = np.zeros((X.shape[0], len(clfs)))\n",
    "    dataset_blend_test = np.zeros((X_submission.shape[0], len(clfs)))\n",
    "\n",
    "    for j, clf in enumerate(clfs):\n",
    "        print j, clf\n",
    "        dataset_blend_test_j = np.zeros((X_submission.shape[0], len(skf)))\n",
    "        for i, (train, test) in enumerate(skf):\n",
    "            print \"Fold\", i\n",
    "            X_train = X.[train]\n",
    "            y_train = y[train]\n",
    "            X_test = X[test]\n",
    "            y_test = y[test]\n",
    "            clf.fit(X_train, y_train)\n",
    "            y_submission = clf.predict_proba(X_test)[:, 1]\n",
    "            dataset_blend_train[test, j] = y_submission\n",
    "            dataset_blend_test_j[:, i] = clf.predict_proba(X_submission)[:, 1]\n",
    "        dataset_blend_test[:, j] = dataset_blend_test_j.mean(1)\n",
    "\n",
    "    print\n",
    "    print \"Blending.\"\n",
    "    clf = LogisticRegression()\n",
    "    clf.fit(dataset_blend_train, y)\n",
    "    y_submission = clf.predict_proba(dataset_blend_test)[:, 1]\n",
    "\n",
    "    print \"Linear stretch of predictions to [0,1]\"\n",
    "    y_submission = (y_submission - y_submission.min()) / (y_submission.max() - y_submission.min())\n",
    "\n",
    "    print \"Saving Results.\"\n",
    "    tmp = np.vstack([range(1, len(y_submission)+1), y_submission]).T\n",
    "    np.savetxt(fname='submission.csv', X=tmp, fmt='%d,%0.9f',\n",
    "               header='MoleculeId,PredictedProbability', comments='')\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
